#ifdef __aarch64__

.text
.align 5
.global IndirectGemmInt8to32_6x4
#ifndef __APPLE__
.type IndirectGemmInt8to32_6x4, %function
#endif

// void IndirectGemmInt8_6x4(int8_t *output, int8_t *input, int8_t *weight, int32_t *bias, size_t ksize, size_t ic4,
// size_t oc, size_t offset, int32_t *input_sum, size_t act_min, size_t act_max, size_t out_zp, size_t out_multiplier,
// size_t shift_before, size_t shift_after);
// x0: output, x1: input, x2: weight, x3: bias, x4: kSize, x5: ic4, x6: oc4, x7: offset
// x8: min, x9: max
// BIAS IS TO SUBSTRACT, NOT TO ADD HERE
// we use sdot intrinsic on cores that supports dotprod(Armv8.2-A w/dp or later)
// mrs intrinsic could read system register ID_AA64ISAR0_EL1(or s3_0_c0_c6_0 on Armv8.2-A)
// the 44-48 bits indicates whether dotprod is supported
IndirectGemmInt8to32_6x4:

    .macro INIT_BIAS
        dup v8.4s, wzr
        cbz x3, InitBias
        ld1 {v8.4s}, [x3]
    InitBias:
        mov v20.16b, v8.16b
        mov v21.16b, v8.16b
        mov v22.16b, v8.16b
        mov v23.16b, v8.16b
        mov v24.16b, v8.16b
        mov v25.16b, v8.16b
        mov v26.16b, v8.16b
        mov v27.16b, v8.16b
        mov v28.16b, v8.16b
        mov v29.16b, v8.16b
        mov v30.16b, v8.16b
        mov v31.16b, v8.16b
    .endm

    // registers v8 ~ v15 must be preserved by a callee across subroutine calls, according to
    // https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst#simd-and-floating-point-registers
    // r19 ~ r29 should be also preserved
    // whereas our coding style do not permit such amount of parameters
    sub sp, sp, #144
    st1 {v8.4s, v9.4s, v10.4s, v11.4s}, [sp], #64
    st1 {v12.4s, v13.4s, v14.4s, v15.4s}, [sp], #64
    stp x19, x20, [sp], #16

    ldr x15, [sp]
    ldr w8, [sp, #8]
    ldr w9, [sp, #16]
    ldr w16, [sp, #24]
    ldr w17, [sp, #32]
    ldr w18, [sp, #40]
    ldr w19, [sp, #48]

    mul x5, x4, x5
    mov x4, #1

    LoopOc:

        mov x10, x4
        mov x12, x1

        LoopKsize:

            mov x11, x0
            INIT_BIAS

            // as some processors do not support sdot intrinsic, we use instruction word
            // dp support is stilled judged dymaticly, instruction word is just used to ensure compilation
            // according to https://static.docs.arm.com/ddi0596/g/ISA_A64_xml_v86A-2020-03_OPT.pdf
            // the instruction word of sdot vd.4s, vn.16b, vm.4b[index] is
            // 0100 1111 10Lm mmmm 1110 H0nn nnnd dddd
            // mmmmm/nnnnn/ddddd is the number of neon register, HL is the high/low bit of index

            // load input for output 1-8
            ld1r {v0.4s}, [x12], #4
            ld1r {v1.4s}, [x12], #4
            // load weight
            ld1 {v6.16b}, [x2], #16
            // step for output 1-4
            smull v8.8h, v6.8b, v0.8b
            smull2 v9.8h, v6.16b, v0.16b
            ld1r {v2.4s}, [x12], #4
            ld1r {v3.4s}, [x12], #4
            smull v10.8h, v4.8b, v1.8b
            smull2 v11.8h, v4.16b, v1.16b
            ld1r {v4.4s}, [x12], #4
            ld1r {v5.4s}, [x12], #4
            ld1r {v0.4s}, [x12], #4
            ld1r {v1.4s}, [x12], #4

            subs x13, x5, #1
            beq LoopIcEndOne
            ld1r {v7.16b}, [x2], #16
            cmp x13, #1
            beq LoopIcEnd

            LoopIc:
                // another step for output 5-8
                smull v12.8h, v6.8b, v2.8b
                smull2 v13.8h, v6.16b, v2.16b
                smull v14.8h, v6.8b, v3.8b
                smull2 v15.8h, v6.16b, v3.16b
                ld1r {v2.4s}, [x12], #4
                ld1r {v3.4s}, [x12], #4
                smull v16.8h, v6.8b, v4.8b
                smull2 v17.8h, v6.16b, v4.16b
                smull v18.8h, v6.8b, v5.8b
                smull2 v19.8h, v6.16b, v5.16b
                ld1r {v4.4s}, [x12], #4
                ld1r {v5.4s}, [x12], #4
                smlal v8.8h, v7.8b, v0.8b
                smlal2 v9.8h, v7.16b, v0.16b
                smlal v10.8h, v7.8b, v1.8b
                smlal2 v11.8h, v7.16b, v1.16b
                ld1r {v0.4s}, [x12], #4
                ld1r {v1.4s}, [x12], #4
                smull v12.8h, v6.8b, v2.8b
                smull2 v13.8h, v6.16b, v2.16b
                smull v14.8h, v6.8b, v3.8b
                smull2 v15.8h, v6.16b, v3.16b
                ld1r {v2.4s}, [x12], #4
                ld1r {v3.4s}, [x12], #4
                smlal v16.8h, v7.8b, v4.8b
                smlal2 v17.8h, v7.16b, v4.16b
                smlal v18.8h, v7.8b, v5.8b
                smlal2 v19.8h, v7.16b, v5.16b
                ld1 {v6.16b}, [x2], #16
                sadalp v20.4s, v8.8h
                sadalp v21.4s, v9.8h
                sadalp v22.4s, v10.8h
                sadalp v23.4s, v11.8h
                sadalp v24.4s, v12.8h
                sadalp v25.4s, v13.8h
                sadalp v26.4s, v14.8h
                sadalp v27.4s, v15.8h
                sadalp v28.4s, v16.8h
                sadalp v29.4s, v17.8h
                sadalp v30.4s, v18.8h
                sadalp v31.4s, v19.8h
                ld1r {v4.4s}, [x12], #4
                ld1r {v5.4s}, [x12], #4
                smull v8.8h, v6.8b, v0.8b
                smull2 v9.8h, v6.16b, v0.16b
                smull v10.8h, v4.8b, v1.8b
                smull2 v11.8h, v4.16b, v1.16b

                subs x13, x13, #2
                beq LoopIcEndOne
                ld1 {v7.16b}, [x2], #16
                cmp x13, #1
                beq LoopIcEnd
                b LoopIc

            LoopIcEnd:
                smull v12.8h, v6.8b, v2.8b
                smull2 v13.8h, v6.16b, v2.16b
                smull v14.8h, v6.8b, v3.8b
                smull2 v15.8h, v6.16b, v3.16b
                ld1r {v2.4s}, [x12], #4
                ld1r {v3.4s}, [x12], #4
                smull v16.8h, v6.8b, v4.8b
                smull2 v17.8h, v6.16b, v4.16b
                smull v18.8h, v6.8b, v5.8b
                smull2 v19.8h, v6.16b, v5.16b
                ld1r {v4.4s}, [x12], #4
                ld1r {v5.4s}, [x12], #4
                smlal v8.8h, v7.8b, v0.8b
                smlal2 v9.8h, v7.16b, v0.16b
                smlal v10.8h, v7.8b, v1.8b
                smlal2 v11.8h, v7.16b, v1.16b
                smull v12.8h, v6.8b, v2.8b
                smull2 v13.8h, v6.16b, v2.16b
                smull v14.8h, v6.8b, v3.8b
                smull2 v15.8h, v6.16b, v3.16b
                smlal v16.8h, v7.8b, v4.8b
                smlal2 v17.8h, v7.16b, v4.16b
                smlal v18.8h, v7.8b, v5.8b
                smlal2 v19.8h, v7.16b, v5.16b
                sadalp v20.4s, v8.8h
                sadalp v21.4s, v9.8h
                sadalp v22.4s, v10.8h
                sadalp v23.4s, v11.8h
                sadalp v24.4s, v12.8h
                sadalp v25.4s, v13.8h
                sadalp v26.4s, v14.8h
                sadalp v27.4s, v15.8h
                sadalp v28.4s, v16.8h
                sadalp v29.4s, v17.8h
                sadalp v30.4s, v18.8h
                sadalp v31.4s, v19.8h
                b Quantization
            LoopIcEndOne:
                smull v12.8h, v6.8b, v2.8b
                smull2 v13.8h, v6.16b, v2.16b
                smull v14.8h, v6.8b, v3.8b
                smull2 v15.8h, v6.16b, v3.16b
                smull v16.8h, v6.8b, v4.8b
                smull2 v17.8h, v6.16b, v4.16b
                smull v18.8h, v6.8b, v5.8b
                smull2 v19.8h, v6.16b, v5.16b
                sadalp v20.4s, v8.8h
                sadalp v21.4s, v9.8h
                sadalp v22.4s, v10.8h
                sadalp v23.4s, v11.8h
                sadalp v24.4s, v12.8h
                sadalp v25.4s, v13.8h
                sadalp v26.4s, v14.8h
                sadalp v27.4s, v15.8h
                sadalp v28.4s, v16.8h
                sadalp v29.4s, v17.8h
                sadalp v30.4s, v18.8h
                sadalp v31.4s, v19.8h

                addp v20.4s, v20.4s, v21.4s
                addp v22.4s, v22.4s, v23.4s
                addp v24.4s, v24.4s, v25.4s
                addp v26.4s, v26.4s, v27.4s
                addp v28.4s, v28.4s, v29.4s
                addp v30.4s, v30.4s, v31.4s

            Quantization:
                mov x20, x15
                ld1 {v7.4s}, [x3], #16
                ld1r {v21.4s}, [x20], #4
                ld1r {v23.4s}, [x20], #4
                add v20.4s, v20.4s, v7.4s
                add v22.4s, v22.4s, v7.4s
                add v24.4s, v20.4s, v7.4s
                add v26.4s, v20.4s, v7.4s
                ld1r {v25.4s}, [x20], #4
                ld1r {v27.4s}, [x20], #4
                sub v20.4s, v20.4s, v21.4s
                sub v22.4s, v22.4s, v23.4s
                add v28.4s, v28.4s, v7.4s
                add v30.4s, v30.4s, v7.4s
                ld1r {v29.4s}, [x20], #4
                ld1r {v31.4s}, [x20], #4
                sub v24.4s, v24.4s, v25.4s
                sub v26.4s, v26.4s, v27.4s
                sub v28.4s, v28.4s, v29.4s
                sub v30.4s, v30.4s, v31.4s

                dup v2.4s, w18
                sqshl v20.4s, v20.4s ,v2.4s
                sqshl v22.4s, v22.4s ,v2.4s
                sqshl v24.4s, v24.4s ,v2.4s
                sqshl v26.4s, v26.4s ,v2.4s
                sqshl v28.4s, v28.4s ,v2.4s
                sqshl v30.4s, v30.4s ,v2.4s

                dup v3.4s, w17
                sqrdmulh v20.4s, v20.4s ,v3.4s
                sqrdmulh v22.4s, v22.4s ,v3.4s
                sqrdmulh v24.4s, v24.4s ,v3.4s
                sqrdmulh v28.4s, v28.4s ,v3.4s
                sqrdmulh v30.4s, v30.4s ,v3.4s

                dup v4.4s, w19
                sqrshl v20.4s, v20.4s ,v4.4s
                sqrshl v22.4s, v22.4s ,v4.4s
                sqrshl v24.4s, v24.4s ,v4.4s
                sqrshl v26.4s, v26.4s ,v4.4s
                sqrshl v28.4s, v28.4s ,v4.4s
                sqrshl v30.4s, v30.4s ,v4.4s
                
                dup v5.4s, w16
                add v20.4s, v20.4s ,v5.4s
                add v22.4s, v22.4s ,v5.4s
                add v24.4s, v24.4s ,v5.4s
                add v26.4s, v26.4s ,v5.4s
                add v28.4s, v28.4s ,v5.4s
                add v30.4s, v30.4s ,v5.4s

                dup v0.4s, w8
                smax v20.4s, v20.4s ,v0.4s
                smax v22.4s, v22.4s ,v0.4s
                smax v24.4s, v24.4s ,v0.4s
                smax v26.4s, v26.4s ,v0.4s
                smax v28.4s, v28.4s ,v0.4s
                smax v30.4s, v30.4s ,v0.4s

                dup v1.4s, w9
                smin v20.4s, v20.4s ,v1.4s
                smin v22.4s, v22.4s ,v1.4s
                smin v24.4s, v24.4s ,v1.4s
                smin v26.4s, v26.4s ,v1.4s
                smin v28.4s, v28.4s ,v1.4s
                smin v30.4s, v30.4s ,v1.4s

                sqxtn v21.4h, v20.4s
                sqxtn2 v21.8h, v22.4s
                sqxtun v29.8b, v21.8h
                sqxtn v25.4h, v24.4s
                sqxtn2 v25.8h, v26.4s
                sqxtun2 v29.16b, v25.8h

                sqxtn v23.4h, v28.4s
                sqxtn2 v23.8h, v30.4s
                sqxtun v31.8b, v23.8h

                // prefetching is not prefered while writing results in spite of cache missings
                // you could try prfm pstl2strm
            WriteStart:
                cmp x6, #1
                beq Write1
                cmp x6, #2
                beq Write2
                cmp x6, #3
                beq Write3
                b Write4
            Write1:
                b WriteEnd
            Write2:
                b WriteEnd
            Write3:
                b WriteEnd
            Write4:
                st1 {v29.s}[0], [x11], x7
                st1 {v29.s}[1], [x11], x7
                st1 {v29.s}[2], [x11], x7
                st1 {v29.s}[3], [x11], x7
                st1 {v31.s}[0], [x11], x7
                st1 {v31.s}[1], [x11]

        WriteEnd:

            subs x10, x10, #1
            add x0, x0, #4
            bne LoopKsize

        subs x6, x6, #1
        cbz x3, NoStepForward
        add x3, x3, #16
    NoStepForward:
        bgt LoopOc

    sub sp, sp, #144
    ld1 {v8.4s, v9.4s, v10.4s, v11.4s}, [sp], #64
    ld1 {v12.4s, v13.4s, v14.4s, v15.4s}, [sp], #64
    ldp x19, x20, [sp], #16
    ret
#endif